{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transfer Learning in Keras\n",
    "\n",
    "In this notebook, we'll cover how to load a pre-trained model (in this case, VGGNet19) and finetune it for a new task: detecting hot dogs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.applications.vgg19 import VGG19\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.callbacks import ModelCheckpoint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load the pre-trained VGG19 model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vgg19 = VGG19(include_top=False,\n",
    "              weights='imagenet',\n",
    "              input_shape=(224,224,3),\n",
    "              pooling=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Freeze all the layers in the base VGGNet19 model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<keras.engine.topology.InputLayer at 0x7ff177a59a58>,\n",
       " <keras.layers.convolutional.Conv2D at 0x7ff177a59c18>,\n",
       " <keras.layers.convolutional.Conv2D at 0x7ff177a59d68>,\n",
       " <keras.layers.pooling.MaxPooling2D at 0x7ff177a73e10>,\n",
       " <keras.layers.convolutional.Conv2D at 0x7ff1777cd358>,\n",
       " <keras.layers.convolutional.Conv2D at 0x7ff1777cdd30>,\n",
       " <keras.layers.pooling.MaxPooling2D at 0x7ff1777ef5f8>,\n",
       " <keras.layers.convolutional.Conv2D at 0x7ff177780c50>,\n",
       " <keras.layers.convolutional.Conv2D at 0x7ff1777942b0>,\n",
       " <keras.layers.convolutional.Conv2D at 0x7ff1777a4588>,\n",
       " <keras.layers.convolutional.Conv2D at 0x7ff177747780>,\n",
       " <keras.layers.pooling.MaxPooling2D at 0x7ff177a53da0>,\n",
       " <keras.layers.convolutional.Conv2D at 0x7ff1e84856d8>,\n",
       " <keras.layers.convolutional.Conv2D at 0x7ff1e84510b8>,\n",
       " <keras.layers.convolutional.Conv2D at 0x7ff1e8464eb8>,\n",
       " <keras.layers.convolutional.Conv2D at 0x7ff20115f780>,\n",
       " <keras.layers.pooling.MaxPooling2D at 0x7ff1776e2208>,\n",
       " <keras.layers.convolutional.Conv2D at 0x7ff1776890f0>,\n",
       " <keras.layers.convolutional.Conv2D at 0x7ff177689b70>,\n",
       " <keras.layers.convolutional.Conv2D at 0x7ff17769b668>,\n",
       " <keras.layers.convolutional.Conv2D at 0x7ff17763d0b8>,\n",
       " <keras.layers.pooling.MaxPooling2D at 0x7ff17765f710>]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vgg19.layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 224, 224, 3)       0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, 224, 224, 64)      36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv4 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv4 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv4 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "=================================================================\n",
      "Total params: 20,024,384\n",
      "Trainable params: 20,024,384\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "vgg19.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for layer in vgg19.layers:\n",
    "    layer.trainable = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Add custom classification layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Instantiate the sequential model and add the VGG19 model: \n",
    "model = Sequential()\n",
    "model.add(vgg19)\n",
    "\n",
    "# Add the custom layers atop the VGG19 model: \n",
    "model.add(Flatten(name='flattened'))\n",
    "model.add(Dropout(0.5, name='dropout'))\n",
    "model.add(Dense(2, activation='softmax', name='predictions'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "vgg19 (Model)                (None, 7, 7, 512)         20024384  \n",
      "_________________________________________________________________\n",
      "flattened (Flatten)          (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 2)                 50178     \n",
      "=================================================================\n",
      "Total params: 20,074,562\n",
      "Trainable params: 50,178\n",
      "Non-trainable params: 20,024,384\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Compile the model for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "output_dir = 'model_output/transfer_VGG'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\n",
    "                                 \"/weights.{epoch:02d}.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if not os.path.exists(output_dir):\n",
    "    os.makedirs(output_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prepare the data for training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The dataset is available for download [here](https://www.kaggle.com/dansbecker/hot-dog-not-hot-dog/home). You should download the zipfile and extract the contents into a folder called `hot-dog-not-hot-dog` in the `notebooks` directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Instantiate two image generator classes:\n",
    "train_datagen = ImageDataGenerator(\n",
    "    rescale=1.0/255,\n",
    "    data_format='channels_last',\n",
    "    rotation_range=30,\n",
    "    horizontal_flip=True,\n",
    "    fill_mode='reflect')\n",
    "\n",
    "valid_datagen = ImageDataGenerator(\n",
    "    rescale=1.0/255,\n",
    "    data_format='channels_last')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define the batch size:\n",
    "batch_size=32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 498 images belonging to 2 classes.\n",
      "Found 500 images belonging to 2 classes.\n"
     ]
    }
   ],
   "source": [
    "# Define the train and validation generators: \n",
    "train_generator = train_datagen.flow_from_directory(\n",
    "    directory='./hot-dog-not-hot-dog/train',\n",
    "    target_size=(224, 224),\n",
    "    classes=['hot_dog','not_hot_dog'],\n",
    "    class_mode='categorical',\n",
    "    batch_size=batch_size,\n",
    "    shuffle=True,\n",
    "    seed=42)\n",
    "\n",
    "valid_generator = valid_datagen.flow_from_directory(\n",
    "    directory='./hot-dog-not-hot-dog/test',\n",
    "    target_size=(224, 224),\n",
    "    classes=['hot_dog','not_hot_dog'],\n",
    "    class_mode='categorical',\n",
    "    batch_size=batch_size,\n",
    "    shuffle=True,\n",
    "    seed=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/16\n",
      "15/15 [==============================] - 66s - loss: 0.9554 - acc: 0.6125 - val_loss: 0.6407 - val_acc: 0.6813\n",
      "Epoch 2/16\n",
      "15/15 [==============================] - 6s - loss: 0.6735 - acc: 0.6653 - val_loss: 0.4638 - val_acc: 0.7708\n",
      "Epoch 3/16\n",
      "15/15 [==============================] - 8s - loss: 0.4699 - acc: 0.7749 - val_loss: 0.4216 - val_acc: 0.8034\n",
      "Epoch 4/16\n",
      "15/15 [==============================] - 7s - loss: 0.4460 - acc: 0.7905 - val_loss: 0.4306 - val_acc: 0.8034\n",
      "Epoch 5/16\n",
      "15/15 [==============================] - 8s - loss: 0.3581 - acc: 0.8359 - val_loss: 0.4347 - val_acc: 0.8013\n",
      "Epoch 6/16\n",
      "15/15 [==============================] - 7s - loss: 0.3165 - acc: 0.8560 - val_loss: 0.4827 - val_acc: 0.7415\n",
      "Epoch 7/16\n",
      "15/15 [==============================] - 8s - loss: 0.3269 - acc: 0.8425 - val_loss: 0.5908 - val_acc: 0.7628\n",
      "Epoch 8/16\n",
      "15/15 [==============================] - 8s - loss: 0.3220 - acc: 0.8461 - val_loss: 0.5413 - val_acc: 0.7692\n",
      "Epoch 9/16\n",
      "15/15 [==============================] - 8s - loss: 0.2519 - acc: 0.8822 - val_loss: 0.5079 - val_acc: 0.7778\n",
      "Epoch 10/16\n",
      "15/15 [==============================] - 7s - loss: 0.3160 - acc: 0.8628 - val_loss: 0.4417 - val_acc: 0.7885\n",
      "Epoch 11/16\n",
      "15/15 [==============================] - 8s - loss: 0.3001 - acc: 0.8729 - val_loss: 0.5363 - val_acc: 0.7714\n",
      "Epoch 12/16\n",
      "15/15 [==============================] - 7s - loss: 0.3297 - acc: 0.8410 - val_loss: 0.5267 - val_acc: 0.7927\n",
      "Epoch 13/16\n",
      "15/15 [==============================] - 8s - loss: 0.2733 - acc: 0.8780 - val_loss: 0.4656 - val_acc: 0.8013\n",
      "Epoch 14/16\n",
      "15/15 [==============================] - 7s - loss: 0.2471 - acc: 0.8899 - val_loss: 0.6002 - val_acc: 0.7842\n",
      "Epoch 15/16\n",
      "15/15 [==============================] - 7s - loss: 0.2397 - acc: 0.8932 - val_loss: 0.5060 - val_acc: 0.7949\n",
      "Epoch 16/16\n",
      "15/15 [==============================] - 8s - loss: 0.2694 - acc: 0.8958 - val_loss: 0.9579 - val_acc: 0.7179\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7ff1774b07f0>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit_generator(train_generator, steps_per_epoch=15, \n",
    "                    epochs=16, validation_data=valid_generator, \n",
    "                    validation_steps=15, callbacks=[modelcheckpoint])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights('model_output/transfer_VGG/weights.02.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
